#!/usr/bin/perl
use warnings;
use strict;
use DB_File;
use Carp::Assert;
use File::Spec::Functions;
use File::Temp qw(tempfile);
use Cwd qw(abs_path);
use Readonly;
Readonly my $dmoz_dir  =>
'/home/xqy/dmoz/Top/Computers/';
Readonly my $min_pages => 300;
Readonly my $max_pages => 2000;
Readonly my $max_links => 5;

our $top_dir = shift || $dmoz_dir;

chdir $dmoz_dir or die "$!";

our @parents = ($top_dir);

our %dirs;

our $count = 0;

&process($top_dir);

while (my ($dir, $pair) = each %dirs) {
    next if $dir =~ m{/Top/News/};
    my ($links_ref, $pages) = @$pair;
    my @links = defined $links_ref ? @$links_ref : ();
    if ($pages > $min_pages && $pages < $max_pages && (!@links || @links < $max_links)) {
        print "$dir\n$pages\n"; 
        print $_, "\n" for @links;
        print "\n\n";
    }
}

untie %dirs;

print STDERR "Done! $count directories scanned.\n";

sub process {
  my $dir = shift;
  $count++;
  print STDERR "Record $count\n" unless $count % 100;
  opendir DIR, $dir or die "$!";
  foreach my $entry (readdir DIR) {
    next if $entry =~ /^\.{1,2}$/;
    $entry = catfile($dir, $entry);
    if (-l $entry) {
      assert($entry =~ /\@$/, $entry);
         my $abs = File::Spec->rel2abs(readlink($entry), $dir);
        my $dst = abs_path($abs);
    foreach my $parent (@parents) {
        last if substr($dst, 0, length($parent)) eq $parent;
        $dirs{$parent}[0] ||= [];
	    push @{$dirs{$parent}[0]}, "$entry => $dst";
      }
    }
    elsif (-d $entry) {
      unshift @parents, $entry;
      &process($entry);
      shift @parents;
   } else {
      assert($entry =~ /\/list$/, $entry);
      my ($count) = split(/\s+/, `wc -l "$entry"`);
      foreach my $parent (@parents) {
          $dirs{$parent}[1] += $count;
      }
    }
  }
  closedir DIR;
}
